﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
using ReBasic.Model;

namespace ReBasic.Helper
{
    public class TermExtractor
    {
        StreamReader sr;
        StreamWriter sw;
        FileStream fsr, fsw;

        int sent_cnt;
        
        public event MainFrame.TripleHandler ProcessEvent;
        public event MainFrame.InfoHandler StateEvent;

        List<SortTerm> lst;
        public static HashSet<String> StopWordSet = new HashSet<string>();

        #region Events
        void OnProcessEvent(int va, int vb, int perc)
        {
            if (ProcessEvent != null)
                ProcessEvent(va, vb, perc);
        }

        void OnStateEvent(string info, bool isOver)
        {
            if (StateEvent != null)
                StateEvent(info, isOver);
        }
        #endregion

        public void Modify()
        {
            int cnt = 0;
            fsr = new FileStream(MainFrame.DataPath + "Term_ori.txt", FileMode.Open);
            sr = new StreamReader(fsr, Encoding.UTF8);
            fsw = new FileStream(MainFrame.TermPath, FileMode.Create);
            sw = new StreamWriter(fsw, Encoding.UTF8);
            while (!sr.EndOfStream)
            {
                string str = sr.ReadLine();
                sw.WriteLine(str);
                sr.ReadLine();
                if (++cnt % 100000 == 0)
                    OnProcessEvent(cnt, cnt, (int)(100.0 * cnt / MainFrame.TotalSent));
            }
            sr.Close(); sw.Close();
            fsr.Close(); fsw.Close();
            OnStateEvent("complete.", true);
        }

        /// <summary>
        /// Put StopWordList to memory
        /// </summary>
        public static void InitStopWord()
        { 
            if (StopWordSet.Count != 0)
                return;
            String StopWordPath = MainFrame.DataPath + "StopWordsList.txt";
            FileStream fsr = new FileStream(StopWordPath, FileMode.Open);
            StreamReader sr = new StreamReader(fsr, Encoding.UTF8);
            while (!sr.EndOfStream)
                StopWordSet.Add(sr.ReadLine());
            sr.Close();
            fsr.Close();
        }

        /// <summary>
        /// 从sent的curBg到curEd，组成一个candidate term，看它是不是term
        /// </summary>
        /// <param name="sent">当前分析的text part，被$#分隔过</param>
        /// <param name="startPoint">text part在sentence中的起始坐标</param>
        /// <param name="length">text part的长度</param>
        /// <param name="SubConDict">Sub-concept Dictionary</param>
        /// <param name="DistTermDict">存放sent中不重复的terms</param>
        void Ext_Kernel_BFS(string[] sent, int startPoint, int length, 
            Dictionary<string, int> SubConDict, HashSet<string> DistTermDict)
        {
            int i, j, l, rt;
            StringBuilder sb;
            lst = new List<SortTerm>();
            for (i = 0; i < length; i++)
            {
                sb = new StringBuilder(sent[startPoint + i]);
                if (SubConDict.ContainsKey(sb.ToString()))
                    lst.Add(new SortTerm(sb.ToString(), i, i));
                for (j = 1; i + j < length; j++)
                {
                    sb.Append(' ');
                    sb.Append(sent[startPoint + i + j]);
                    if (SubConDict.ContainsKey(sb.ToString()))
                        lst.Add(new SortTerm(sb.ToString(), i, i + j));
                }
            }
            lst.Sort();
            for (i = 0, l = lst.Count, rt = -1; i < l; i++)     //Get longest cover.
                if (lst[i].ed > rt)
                {
                    rt = lst[i].ed;
                    if (!StopWordSet.Contains(lst[i].term))     //Not Stop words
                        DistTermDict.Add(lst[i].term);
                }
        }

        /// <summary>
        /// Extract distinct terms which are in Probase from a sentence
        /// </summary>
        /// <param name="sent">Finding sentence</param>
        /// <param name="SubConDict">Dictionary of Probase sub-concepts</param>
        /// <param name="result">Set to store terms, can be null.</param>
        public void ExtractTerm(string sent, Dictionary<string, int> SubConDict, ref List<string> result)
        {
            HashSet<string> DistTermDict = new HashSet<string>();        //distinct terms in this sentence
            bool[] msk = new bool[65536];
            bool optFile = (result == null ? true : false);     //whether to output to file
            msk[','] = msk['$'] = msk['#'] = msk['='] 
                = msk['-'] = msk['\\'] = true;                  //Using these chars to split continuous part of terms
            int bg, ed, len;
            string[] spt = sent.Split(new char[] {' '}, StringSplitOptions.RemoveEmptyEntries);
            bg = ed = 0;
            len = spt.Length;                  //Whether term is found in this sentence
            #region Get continuous part of terms
            while (bg < len && ed < len)
            {
                if (msk[spt[ed][0]] == true)       //found spliter
                {
                    if (bg <= ed - 1)
                        Ext_Kernel_BFS(spt, bg, ed - bg, SubConDict, DistTermDict);
                    ed++;
                    bg = ed;
                }
                else
                    ed++;
            }
            if (bg <= ed - 1)
                Ext_Kernel_BFS(spt, bg, ed - bg, SubConDict, DistTermDict);
            #endregion
            #region write terms
            if (optFile == true)
            {
                List<string> tmpList = DistTermDict.ToList();
                for (int i = 0; i < tmpList.Count; i++)
                {
                    if (i > 0) sw.Write('\t');
                    sw.Write(tmpList[i]);
                }
                sw.WriteLine();
            }
            else
            {
                result.Clear();
                foreach (string s in DistTermDict)
                    result.Add(s);
            }
            #endregion
        }

        public void Work(Dictionary<string, int> SubConDict)
        {
            List<string> result = null;     //占位符...我瞎了
            sent_cnt = 0;
            fsr = new FileStream(MainFrame.SentPath, FileMode.Open);
            fsw = new FileStream(MainFrame.TermPath, FileMode.Create);
            sr = new StreamReader(fsr, Encoding.UTF8);
            sw = new StreamWriter(fsw, Encoding.UTF8);
            while (!sr.EndOfStream)
            {
                string str = sr.ReadLine();
                ExtractTerm(str, SubConDict, ref result);
                if (++sent_cnt % 100000 == 0)
                    OnProcessEvent(MainFrame.TotalSent, sent_cnt,
                        (int)(100.0 * sent_cnt / MainFrame.TotalSent));
            }
            sr.Close(); sw.Close();
            fsr.Close(); fsw.Close();
            OnStateEvent("Generateing Term_Ds complete, with " 
                + sent_cnt.ToString() + " sentences", true);
        }

        public void CloseFile(int whichWork)
        {
            if (sr != null)
            {
                sr.Close();
                fsr.Close();
            }
            if (sw != null)
            {
                sw.Close();
                fsw.Close();
            }
            OnStateEvent("Generateing Term_Ds stopped, with "
                + sent_cnt.ToString() + " sentences.", true);
        }
    }
}
